#!/usr/python
# --* coding:utf-8 *--
#
import os
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
import re

import scrapy
from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors import LinkExtractor

from learnscrapy.items import LearnscrapyItem, X23sContentItem
class X23usSpider(scrapy.Spider):
    name = 'x23us'
    allowed_domains = ["x23us.com"]
    start_urls = []

    def start_requests(self):
        url_head = "http://www.x23us.com/quanben/"
        for p in range(1, 792):
            self.start_urls.append( url_head + str(p) )

        for url in self.start_urls:
            yield self.make_requests_from_url(url)

    def parse(self, response):
        lists = response.css("table tr")
        learnitems = []
        for item in lists:
            if item.css("td:nth-child(1)"):
                learnitem = LearnscrapyItem()
                learnitem['name'] = item.css("td:nth-child(1) a:nth-child(2)::text").extract_first()
                book_url = item.css("td:nth-child(1) a:nth-child(2)::attr(href)").extract_first()
                pattern = re.compile(r"[1-9]\d*")
                book_id = re.findall(pattern,book_url)[-1]

                learnitem['author'] = item.css("td:nth-child(3)::text").extract_first()
                learnitem['date'] = "20"+item.css("td:nth-child(5)::text").extract_first()
                learnitem['bid'] = book_id

                yield learnitem

                if book_url:
                    yield scrapy.Request(url=book_url, callback=self.parse_title, meta={'bid': book_id})

    def parse_title(self, response):
        bid = response.meta['bid']
        base_url = response.url
        titles_html = response.css("#at td")
        if titles_html:
            for item in titles_html:
                if item.css("a"):
                    url = base_url+item.css("a::attr(href)").extract_first()
                    title =item.css("a::text").extract_first()
                    yield scrapy.Request( url, callback=self.parse_content, meta={'bid': bid, 'title': title, 'url': url} )

    def parse_content(self, response):
        content = response.css("#contents::text").extract_first()
        x23us_item =X23sContentItem()
        x23us_item['content'] = content
        x23us_item['title'] = response.meta['title']
        x23us_item['bid'] = response.meta['bid']
        x23us_item['url'] = response.meta['url']
        return x23us_item
